{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PyTorch 实现语言模型的demo\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2-gram\n",
    "CONTEXT_SIZE = 2\n",
    "# We will use Shakespeare Sonnet 2\n",
    "test_sentence = \"\"\"When forty winters shall besiege thy brow,\n",
    "And dig deep trenches in thy beauty's field,\n",
    "Thy youth's proud livery so gazed on now,\n",
    "Will be a totter'd weed of small worth held:\n",
    "Then being asked, where all thy beauty lies,\n",
    "Where all the treasure of thy lusty days;\n",
    "To say, within thine own deep sunken eyes,\n",
    "Were an all-eating shame, and thriftless praise.\n",
    "How much more praise deserv'd thy beauty's use,\n",
    "If thou couldst answer 'This fair child of mine\n",
    "Shall sum my count, and make my old excuse,'\n",
    "Proving his beauty by succession thine!\n",
    "This were to be new made when thou art old,\n",
    "And see thy blood warm when thou feel'st it cold.\"\"\".split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 给每个单词编码，也就是用数字来表示每个单词，这样才能够传入word embeding得到词向量。\n",
    "vocab = set(test_sentence) # 通过set将重复的单词去掉\n",
    "word_to_idx = {word: i+1 for i, word in enumerate(vocab)}\n",
    "# 定义了一个unknown的词，也就是说没有出现在训练集里的词，我们都叫做unknown，词向量就定义为0。\n",
    "word_to_idx['<unk>'] = 0\n",
    "idx_to_word = {i+1: word for i, word in enumerate(vocab)}\n",
    "idx_to_word[0] = '<unk>'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将数据整理好，也就是我们需要将单词三个分组，每个组前两个作为传入的数据，而最后一个作为预测的结果。\n",
    "trigram = [((test_sentence[i], test_sentence[i+1]), test_sentence[i+2])\n",
    "           for i in range(len(test_sentence)-2)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 使用(加载)预训练的词向量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "wvmodel = gensim.models.KeyedVectors.load_word2vec_format('/Users/wyw/Documents/vectors/word2vec/word2vec.6B.100d.txt', binary=False, encoding='utf-8')\n",
    "\n",
    "vocab_size = len(word_to_idx)\n",
    "embed_size = 100\n",
    "weight = torch.zeros(vocab_size, embed_size)\n",
    "\n",
    "for i in range(len(wvmodel.index2word)):\n",
    "    try:\n",
    "        index = word_to_idx[wvmodel.index2word[i]]\n",
    "    except:\n",
    "        continue\n",
    "    weight[index, :] = torch.from_numpy(wvmodel.get_vector(\n",
    "        idx_to_word[word_to_idx[wvmodel.index2word[i]]]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 定义模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NgramModel(nn.Module):\n",
    "    def __init__(self, vocb_size, context_size, n_dim):\n",
    "        super(NgramModel, self).__init__()\n",
    "        self.n_word = vocb_size\n",
    "        \n",
    "        # 在Embedding层中不使用预训练好的word2vec词向量\n",
    "        # self.embedding = nn.Embedding(self.n_word, n_dim)\n",
    "        \n",
    "        # 使用预训练词向量\n",
    "        self.embedding = nn.Embedding.from_pretrained(weight)\n",
    "        # requires_grad指定是否在训练过程中对词向量的权重进行微调\n",
    "        self.embedding.weight.requires_grad = True\n",
    "        \n",
    "        self.linear1 = nn.Linear(context_size*n_dim, 128)\n",
    "        self.linear2 = nn.Linear(128, self.n_word)\n",
    "\n",
    "    def forward(self, x):\n",
    "        emb = self.embedding(x)\n",
    "        emb = emb.view(1, -1)\n",
    "        out = self.linear1(emb)\n",
    "        out = F.relu(out)\n",
    "        out = self.linear2(out)\n",
    "        log_prob = F.log_softmax(out)\n",
    "        return log_prob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "ngrammodel = NgramModel(len(word_to_idx), CONTEXT_SIZE, 100)\n",
    "criterion = nn.NLLLoss()\n",
    "optimizer = optim.SGD(ngrammodel.parameters(), lr=1e-3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 开始训练"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 一共跑300个epoch，在每个epoch中，word代表着预测单词的前面两个词，label表示要预测的词，接着进入网络得到结果，然后通过loss函数得到loss进行反向传播，更新参数。\n",
    "- 此例中，使用预训练的词向量在训练过程中的收敛速度会远慢于不使用预训练词向量，这是因为样本太小，不使用预训练词向量的时候会很更快的达到过拟合，所以损失值会减小的更快。可以动手试一下，也可以改一下self.embedding.weight.requires_grad = False，看一下在训练过程中微调词向量能否带来影响,但是因为样本太小，估计也看不出啥影响 =__= "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1\n",
      "**********\n",
      "Loss: 5.299686\n",
      "epoch: 2\n",
      "**********\n",
      "Loss: 5.287130\n",
      "epoch: 3\n",
      "**********\n",
      "Loss: 5.274845\n",
      "epoch: 4\n",
      "**********\n",
      "Loss: 5.262775\n",
      "epoch: 5\n",
      "**********\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/envs/pytorch/lib/python3.6/site-packages/ipykernel/__main__.py:23: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "/anaconda3/envs/pytorch/lib/python3.6/site-packages/ipykernel/__main__.py:12: UserWarning: invalid index of a 0-dim tensor. This will be an error in PyTorch 0.5. Use tensor.item() to convert a 0-dim tensor to a Python number\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss: 5.250835\n",
      "epoch: 6\n",
      "**********\n",
      "Loss: 5.238929\n",
      "epoch: 7\n",
      "**********\n",
      "Loss: 5.227102\n",
      "epoch: 8\n",
      "**********\n",
      "Loss: 5.215246\n",
      "epoch: 9\n",
      "**********\n",
      "Loss: 5.203434\n",
      "epoch: 10\n",
      "**********\n",
      "Loss: 5.191671\n",
      "epoch: 11\n",
      "**********\n",
      "Loss: 5.179883\n",
      "epoch: 12\n",
      "**********\n",
      "Loss: 5.168078\n",
      "epoch: 13\n",
      "**********\n",
      "Loss: 5.156142\n",
      "epoch: 14\n",
      "**********\n",
      "Loss: 5.144120\n",
      "epoch: 15\n",
      "**********\n",
      "Loss: 5.131994\n",
      "epoch: 16\n",
      "**********\n",
      "Loss: 5.119658\n",
      "epoch: 17\n",
      "**********\n",
      "Loss: 5.107174\n",
      "epoch: 18\n",
      "**********\n",
      "Loss: 5.094490\n",
      "epoch: 19\n",
      "**********\n",
      "Loss: 5.081558\n",
      "epoch: 20\n",
      "**********\n",
      "Loss: 5.068453\n",
      "epoch: 21\n",
      "**********\n",
      "Loss: 5.055124\n",
      "epoch: 22\n",
      "**********\n",
      "Loss: 5.041547\n",
      "epoch: 23\n",
      "**********\n",
      "Loss: 5.027707\n",
      "epoch: 24\n",
      "**********\n",
      "Loss: 5.013658\n",
      "epoch: 25\n",
      "**********\n",
      "Loss: 4.999541\n",
      "epoch: 26\n",
      "**********\n",
      "Loss: 4.985229\n",
      "epoch: 27\n",
      "**********\n",
      "Loss: 4.970750\n",
      "epoch: 28\n",
      "**********\n",
      "Loss: 4.956045\n",
      "epoch: 29\n",
      "**********\n",
      "Loss: 4.941186\n",
      "epoch: 30\n",
      "**********\n",
      "Loss: 4.926172\n",
      "epoch: 31\n",
      "**********\n",
      "Loss: 4.911053\n",
      "epoch: 32\n",
      "**********\n",
      "Loss: 4.895741\n",
      "epoch: 33\n",
      "**********\n",
      "Loss: 4.880311\n",
      "epoch: 34\n",
      "**********\n",
      "Loss: 4.864707\n",
      "epoch: 35\n",
      "**********\n",
      "Loss: 4.849030\n",
      "epoch: 36\n",
      "**********\n",
      "Loss: 4.833299\n",
      "epoch: 37\n",
      "**********\n",
      "Loss: 4.817461\n",
      "epoch: 38\n",
      "**********\n",
      "Loss: 4.801511\n",
      "epoch: 39\n",
      "**********\n",
      "Loss: 4.785579\n",
      "epoch: 40\n",
      "**********\n",
      "Loss: 4.769629\n",
      "epoch: 41\n",
      "**********\n",
      "Loss: 4.753661\n",
      "epoch: 42\n",
      "**********\n",
      "Loss: 4.737655\n",
      "epoch: 43\n",
      "**********\n",
      "Loss: 4.721619\n",
      "epoch: 44\n",
      "**********\n",
      "Loss: 4.705519\n",
      "epoch: 45\n",
      "**********\n",
      "Loss: 4.689377\n",
      "epoch: 46\n",
      "**********\n",
      "Loss: 4.673133\n",
      "epoch: 47\n",
      "**********\n",
      "Loss: 4.656794\n",
      "epoch: 48\n",
      "**********\n",
      "Loss: 4.640369\n",
      "epoch: 49\n",
      "**********\n",
      "Loss: 4.623907\n",
      "epoch: 50\n",
      "**********\n",
      "Loss: 4.607397\n",
      "epoch: 51\n",
      "**********\n",
      "Loss: 4.590748\n",
      "epoch: 52\n",
      "**********\n",
      "Loss: 4.574033\n",
      "epoch: 53\n",
      "**********\n",
      "Loss: 4.557155\n",
      "epoch: 54\n",
      "**********\n",
      "Loss: 4.540222\n",
      "epoch: 55\n",
      "**********\n",
      "Loss: 4.523139\n",
      "epoch: 56\n",
      "**********\n",
      "Loss: 4.505937\n",
      "epoch: 57\n",
      "**********\n",
      "Loss: 4.488670\n",
      "epoch: 58\n",
      "**********\n",
      "Loss: 4.471297\n",
      "epoch: 59\n",
      "**********\n",
      "Loss: 4.453774\n",
      "epoch: 60\n",
      "**********\n",
      "Loss: 4.436069\n",
      "epoch: 61\n",
      "**********\n",
      "Loss: 4.418253\n",
      "epoch: 62\n",
      "**********\n",
      "Loss: 4.400317\n",
      "epoch: 63\n",
      "**********\n",
      "Loss: 4.382170\n",
      "epoch: 64\n",
      "**********\n",
      "Loss: 4.363972\n",
      "epoch: 65\n",
      "**********\n",
      "Loss: 4.345636\n",
      "epoch: 66\n",
      "**********\n",
      "Loss: 4.327049\n",
      "epoch: 67\n",
      "**********\n",
      "Loss: 4.308388\n",
      "epoch: 68\n",
      "**********\n",
      "Loss: 4.289589\n",
      "epoch: 69\n",
      "**********\n",
      "Loss: 4.270721\n",
      "epoch: 70\n",
      "**********\n",
      "Loss: 4.251706\n",
      "epoch: 71\n",
      "**********\n",
      "Loss: 4.232567\n",
      "epoch: 72\n",
      "**********\n",
      "Loss: 4.213307\n",
      "epoch: 73\n",
      "**********\n",
      "Loss: 4.193932\n",
      "epoch: 74\n",
      "**********\n",
      "Loss: 4.174408\n",
      "epoch: 75\n",
      "**********\n",
      "Loss: 4.154777\n",
      "epoch: 76\n",
      "**********\n",
      "Loss: 4.135058\n",
      "epoch: 77\n",
      "**********\n",
      "Loss: 4.115153\n",
      "epoch: 78\n",
      "**********\n",
      "Loss: 4.095215\n",
      "epoch: 79\n",
      "**********\n",
      "Loss: 4.074976\n",
      "epoch: 80\n",
      "**********\n",
      "Loss: 4.054722\n",
      "epoch: 81\n",
      "**********\n",
      "Loss: 4.034311\n",
      "epoch: 82\n",
      "**********\n",
      "Loss: 4.013821\n",
      "epoch: 83\n",
      "**********\n",
      "Loss: 3.993260\n",
      "epoch: 84\n",
      "**********\n",
      "Loss: 3.972582\n",
      "epoch: 85\n",
      "**********\n",
      "Loss: 3.951818\n",
      "epoch: 86\n",
      "**********\n",
      "Loss: 3.930863\n",
      "epoch: 87\n",
      "**********\n",
      "Loss: 3.909885\n",
      "epoch: 88\n",
      "**********\n",
      "Loss: 3.888734\n",
      "epoch: 89\n",
      "**********\n",
      "Loss: 3.867528\n",
      "epoch: 90\n",
      "**********\n",
      "Loss: 3.846180\n",
      "epoch: 91\n",
      "**********\n",
      "Loss: 3.824765\n",
      "epoch: 92\n",
      "**********\n",
      "Loss: 3.803266\n",
      "epoch: 93\n",
      "**********\n",
      "Loss: 3.781704\n",
      "epoch: 94\n",
      "**********\n",
      "Loss: 3.760009\n",
      "epoch: 95\n",
      "**********\n",
      "Loss: 3.738206\n",
      "epoch: 96\n",
      "**********\n",
      "Loss: 3.716391\n",
      "epoch: 97\n",
      "**********\n",
      "Loss: 3.694468\n",
      "epoch: 98\n",
      "**********\n",
      "Loss: 3.672464\n",
      "epoch: 99\n",
      "**********\n",
      "Loss: 3.650385\n",
      "epoch: 100\n",
      "**********\n",
      "Loss: 3.628392\n",
      "epoch: 101\n",
      "**********\n",
      "Loss: 3.606270\n",
      "epoch: 102\n",
      "**********\n",
      "Loss: 3.584042\n",
      "epoch: 103\n",
      "**********\n",
      "Loss: 3.561761\n",
      "epoch: 104\n",
      "**********\n",
      "Loss: 3.539518\n",
      "epoch: 105\n",
      "**********\n",
      "Loss: 3.517208\n",
      "epoch: 106\n",
      "**********\n",
      "Loss: 3.494924\n",
      "epoch: 107\n",
      "**********\n",
      "Loss: 3.472595\n",
      "epoch: 108\n",
      "**********\n",
      "Loss: 3.450264\n",
      "epoch: 109\n",
      "**********\n",
      "Loss: 3.427943\n",
      "epoch: 110\n",
      "**********\n",
      "Loss: 3.405709\n",
      "epoch: 111\n",
      "**********\n",
      "Loss: 3.383354\n",
      "epoch: 112\n",
      "**********\n",
      "Loss: 3.361176\n",
      "epoch: 113\n",
      "**********\n",
      "Loss: 3.338919\n",
      "epoch: 114\n",
      "**********\n",
      "Loss: 3.316754\n",
      "epoch: 115\n",
      "**********\n",
      "Loss: 3.294584\n",
      "epoch: 116\n",
      "**********\n",
      "Loss: 3.272461\n",
      "epoch: 117\n",
      "**********\n",
      "Loss: 3.250422\n",
      "epoch: 118\n",
      "**********\n",
      "Loss: 3.228392\n",
      "epoch: 119\n",
      "**********\n",
      "Loss: 3.206477\n",
      "epoch: 120\n",
      "**********\n",
      "Loss: 3.184540\n",
      "epoch: 121\n",
      "**********\n",
      "Loss: 3.162734\n",
      "epoch: 122\n",
      "**********\n",
      "Loss: 3.140950\n",
      "epoch: 123\n",
      "**********\n",
      "Loss: 3.119299\n",
      "epoch: 124\n",
      "**********\n",
      "Loss: 3.097675\n",
      "epoch: 125\n",
      "**********\n",
      "Loss: 3.076150\n",
      "epoch: 126\n",
      "**********\n",
      "Loss: 3.054706\n",
      "epoch: 127\n",
      "**********\n",
      "Loss: 3.033343\n",
      "epoch: 128\n",
      "**********\n",
      "Loss: 3.012080\n",
      "epoch: 129\n",
      "**********\n",
      "Loss: 2.990933\n",
      "epoch: 130\n",
      "**********\n",
      "Loss: 2.969876\n",
      "epoch: 131\n",
      "**********\n",
      "Loss: 2.948874\n",
      "epoch: 132\n",
      "**********\n",
      "Loss: 2.928113\n",
      "epoch: 133\n",
      "**********\n",
      "Loss: 2.907380\n",
      "epoch: 134\n",
      "**********\n",
      "Loss: 2.886833\n",
      "epoch: 135\n",
      "**********\n",
      "Loss: 2.866321\n",
      "epoch: 136\n",
      "**********\n",
      "Loss: 2.846051\n",
      "epoch: 137\n",
      "**********\n",
      "Loss: 2.825876\n",
      "epoch: 138\n",
      "**********\n",
      "Loss: 2.805856\n",
      "epoch: 139\n",
      "**********\n",
      "Loss: 2.785951\n",
      "epoch: 140\n",
      "**********\n",
      "Loss: 2.766165\n",
      "epoch: 141\n",
      "**********\n",
      "Loss: 2.746588\n",
      "epoch: 142\n",
      "**********\n",
      "Loss: 2.727098\n",
      "epoch: 143\n",
      "**********\n",
      "Loss: 2.707800\n",
      "epoch: 144\n",
      "**********\n",
      "Loss: 2.688565\n",
      "epoch: 145\n",
      "**********\n",
      "Loss: 2.669508\n",
      "epoch: 146\n",
      "**********\n",
      "Loss: 2.650598\n",
      "epoch: 147\n",
      "**********\n",
      "Loss: 2.631844\n",
      "epoch: 148\n",
      "**********\n",
      "Loss: 2.613271\n",
      "epoch: 149\n",
      "**********\n",
      "Loss: 2.594869\n",
      "epoch: 150\n",
      "**********\n",
      "Loss: 2.576656\n",
      "epoch: 151\n",
      "**********\n",
      "Loss: 2.558593\n",
      "epoch: 152\n",
      "**********\n",
      "Loss: 2.540653\n",
      "epoch: 153\n",
      "**********\n",
      "Loss: 2.522874\n",
      "epoch: 154\n",
      "**********\n",
      "Loss: 2.505352\n",
      "epoch: 155\n",
      "**********\n",
      "Loss: 2.487901\n",
      "epoch: 156\n",
      "**********\n",
      "Loss: 2.470620\n",
      "epoch: 157\n",
      "**********\n",
      "Loss: 2.453578\n",
      "epoch: 158\n",
      "**********\n",
      "Loss: 2.436615\n",
      "epoch: 159\n",
      "**********\n",
      "Loss: 2.419829\n",
      "epoch: 160\n",
      "**********\n",
      "Loss: 2.403217\n",
      "epoch: 161\n",
      "**********\n",
      "Loss: 2.386749\n",
      "epoch: 162\n",
      "**********\n",
      "Loss: 2.370459\n",
      "epoch: 163\n",
      "**********\n",
      "Loss: 2.354347\n",
      "epoch: 164\n",
      "**********\n",
      "Loss: 2.338364\n",
      "epoch: 165\n",
      "**********\n",
      "Loss: 2.322565\n",
      "epoch: 166\n",
      "**********\n",
      "Loss: 2.306921\n",
      "epoch: 167\n",
      "**********\n",
      "Loss: 2.291456\n",
      "epoch: 168\n",
      "**********\n",
      "Loss: 2.276164\n",
      "epoch: 169\n",
      "**********\n",
      "Loss: 2.260997\n",
      "epoch: 170\n",
      "**********\n",
      "Loss: 2.246034\n",
      "epoch: 171\n",
      "**********\n",
      "Loss: 2.231210\n",
      "epoch: 172\n",
      "**********\n",
      "Loss: 2.216569\n",
      "epoch: 173\n",
      "**********\n",
      "Loss: 2.202028\n",
      "epoch: 174\n",
      "**********\n",
      "Loss: 2.187747\n",
      "epoch: 175\n",
      "**********\n",
      "Loss: 2.173556\n",
      "epoch: 176\n",
      "**********\n",
      "Loss: 2.159551\n",
      "epoch: 177\n",
      "**********\n",
      "Loss: 2.145681\n",
      "epoch: 178\n",
      "**********\n",
      "Loss: 2.131995\n",
      "epoch: 179\n",
      "**********\n",
      "Loss: 2.118429\n",
      "epoch: 180\n",
      "**********\n",
      "Loss: 2.105061\n",
      "epoch: 181\n",
      "**********\n",
      "Loss: 2.091804\n",
      "epoch: 182\n",
      "**********\n",
      "Loss: 2.078727\n",
      "epoch: 183\n",
      "**********\n",
      "Loss: 2.065796\n",
      "epoch: 184\n",
      "**********\n",
      "Loss: 2.052957\n",
      "epoch: 185\n",
      "**********\n",
      "Loss: 2.040341\n",
      "epoch: 186\n",
      "**********\n",
      "Loss: 2.027834\n",
      "epoch: 187\n",
      "**********\n",
      "Loss: 2.015461\n",
      "epoch: 188\n",
      "**********\n",
      "Loss: 2.003253\n",
      "epoch: 189\n",
      "**********\n",
      "Loss: 1.991142\n",
      "epoch: 190\n",
      "**********\n",
      "Loss: 1.979218\n",
      "epoch: 191\n",
      "**********\n",
      "Loss: 1.967389\n",
      "epoch: 192\n",
      "**********\n",
      "Loss: 1.955741\n",
      "epoch: 193\n",
      "**********\n",
      "Loss: 1.944195\n",
      "epoch: 194\n",
      "**********\n",
      "Loss: 1.932797\n",
      "epoch: 195\n",
      "**********\n",
      "Loss: 1.921521\n",
      "epoch: 196\n",
      "**********\n",
      "Loss: 1.910363\n",
      "epoch: 197\n",
      "**********\n",
      "Loss: 1.899341\n",
      "epoch: 198\n",
      "**********\n",
      "Loss: 1.888476\n",
      "epoch: 199\n",
      "**********\n",
      "Loss: 1.877707\n",
      "epoch: 200\n",
      "**********\n",
      "Loss: 1.867092\n",
      "epoch: 201\n",
      "**********\n",
      "Loss: 1.856593\n",
      "epoch: 202\n",
      "**********\n",
      "Loss: 1.846245\n",
      "epoch: 203\n",
      "**********\n",
      "Loss: 1.835976\n",
      "epoch: 204\n",
      "**********\n",
      "Loss: 1.825837\n",
      "epoch: 205\n",
      "**********\n",
      "Loss: 1.815835\n",
      "epoch: 206\n",
      "**********\n",
      "Loss: 1.805950\n",
      "epoch: 207\n",
      "**********\n",
      "Loss: 1.796111\n",
      "epoch: 208\n",
      "**********\n",
      "Loss: 1.786483\n",
      "epoch: 209\n",
      "**********\n",
      "Loss: 1.776889\n",
      "epoch: 210\n",
      "**********\n",
      "Loss: 1.767473\n",
      "epoch: 211\n",
      "**********\n",
      "Loss: 1.758119\n",
      "epoch: 212\n",
      "**********\n",
      "Loss: 1.748907\n",
      "epoch: 213\n",
      "**********\n",
      "Loss: 1.739776\n",
      "epoch: 214\n",
      "**********\n",
      "Loss: 1.730768\n",
      "epoch: 215\n",
      "**********\n",
      "Loss: 1.721902\n",
      "epoch: 216\n",
      "**********\n",
      "Loss: 1.713086\n",
      "epoch: 217\n",
      "**********\n",
      "Loss: 1.704388\n",
      "epoch: 218\n",
      "**********\n",
      "Loss: 1.695796\n",
      "epoch: 219\n",
      "**********\n",
      "Loss: 1.687284\n",
      "epoch: 220\n",
      "**********\n",
      "Loss: 1.678875\n",
      "epoch: 221\n",
      "**********\n",
      "Loss: 1.670615\n",
      "epoch: 222\n",
      "**********\n",
      "Loss: 1.662373\n",
      "epoch: 223\n",
      "**********\n",
      "Loss: 1.654319\n",
      "epoch: 224\n",
      "**********\n",
      "Loss: 1.646277\n",
      "epoch: 225\n",
      "**********\n",
      "Loss: 1.638381\n",
      "epoch: 226\n",
      "**********\n",
      "Loss: 1.630571\n",
      "epoch: 227\n",
      "**********\n",
      "Loss: 1.622830\n",
      "epoch: 228\n",
      "**********\n",
      "Loss: 1.615185\n",
      "epoch: 229\n",
      "**********\n",
      "Loss: 1.607655\n",
      "epoch: 230\n",
      "**********\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss: 1.600212\n",
      "epoch: 231\n",
      "**********\n",
      "Loss: 1.592801\n",
      "epoch: 232\n",
      "**********\n",
      "Loss: 1.585529\n",
      "epoch: 233\n",
      "**********\n",
      "Loss: 1.578311\n",
      "epoch: 234\n",
      "**********\n",
      "Loss: 1.571181\n",
      "epoch: 235\n",
      "**********\n",
      "Loss: 1.564126\n",
      "epoch: 236\n",
      "**********\n",
      "Loss: 1.557174\n",
      "epoch: 237\n",
      "**********\n",
      "Loss: 1.550283\n",
      "epoch: 238\n",
      "**********\n",
      "Loss: 1.543477\n",
      "epoch: 239\n",
      "**********\n",
      "Loss: 1.536721\n",
      "epoch: 240\n",
      "**********\n",
      "Loss: 1.530059\n",
      "epoch: 241\n",
      "**********\n",
      "Loss: 1.523502\n",
      "epoch: 242\n",
      "**********\n",
      "Loss: 1.516961\n",
      "epoch: 243\n",
      "**********\n",
      "Loss: 1.510518\n",
      "epoch: 244\n",
      "**********\n",
      "Loss: 1.504134\n",
      "epoch: 245\n",
      "**********\n",
      "Loss: 1.497846\n",
      "epoch: 246\n",
      "**********\n",
      "Loss: 1.491604\n",
      "epoch: 247\n",
      "**********\n",
      "Loss: 1.485444\n",
      "epoch: 248\n",
      "**********\n",
      "Loss: 1.479346\n",
      "epoch: 249\n",
      "**********\n",
      "Loss: 1.473330\n",
      "epoch: 250\n",
      "**********\n",
      "Loss: 1.467381\n",
      "epoch: 251\n",
      "**********\n",
      "Loss: 1.461480\n",
      "epoch: 252\n",
      "**********\n",
      "Loss: 1.455651\n",
      "epoch: 253\n",
      "**********\n",
      "Loss: 1.449876\n",
      "epoch: 254\n",
      "**********\n",
      "Loss: 1.444189\n",
      "epoch: 255\n",
      "**********\n",
      "Loss: 1.438560\n",
      "epoch: 256\n",
      "**********\n",
      "Loss: 1.432970\n",
      "epoch: 257\n",
      "**********\n",
      "Loss: 1.427444\n",
      "epoch: 258\n",
      "**********\n",
      "Loss: 1.421986\n",
      "epoch: 259\n",
      "**********\n",
      "Loss: 1.416596\n",
      "epoch: 260\n",
      "**********\n",
      "Loss: 1.411253\n",
      "epoch: 261\n",
      "**********\n",
      "Loss: 1.405932\n",
      "epoch: 262\n",
      "**********\n",
      "Loss: 1.400736\n",
      "epoch: 263\n",
      "**********\n",
      "Loss: 1.395560\n",
      "epoch: 264\n",
      "**********\n",
      "Loss: 1.390404\n",
      "epoch: 265\n",
      "**********\n",
      "Loss: 1.385343\n",
      "epoch: 266\n",
      "**********\n",
      "Loss: 1.380336\n",
      "epoch: 267\n",
      "**********\n",
      "Loss: 1.375374\n",
      "epoch: 268\n",
      "**********\n",
      "Loss: 1.370442\n",
      "epoch: 269\n",
      "**********\n",
      "Loss: 1.365581\n",
      "epoch: 270\n",
      "**********\n",
      "Loss: 1.360776\n",
      "epoch: 271\n",
      "**********\n",
      "Loss: 1.356003\n",
      "epoch: 272\n",
      "**********\n",
      "Loss: 1.351286\n",
      "epoch: 273\n",
      "**********\n",
      "Loss: 1.346628\n",
      "epoch: 274\n",
      "**********\n",
      "Loss: 1.341983\n",
      "epoch: 275\n",
      "**********\n",
      "Loss: 1.337422\n",
      "epoch: 276\n",
      "**********\n",
      "Loss: 1.332881\n",
      "epoch: 277\n",
      "**********\n",
      "Loss: 1.328399\n",
      "epoch: 278\n",
      "**********\n",
      "Loss: 1.323950\n",
      "epoch: 279\n",
      "**********\n",
      "Loss: 1.319548\n",
      "epoch: 280\n",
      "**********\n",
      "Loss: 1.315218\n",
      "epoch: 281\n",
      "**********\n",
      "Loss: 1.310891\n",
      "epoch: 282\n",
      "**********\n",
      "Loss: 1.306637\n",
      "epoch: 283\n",
      "**********\n",
      "Loss: 1.302406\n",
      "epoch: 284\n",
      "**********\n",
      "Loss: 1.298215\n",
      "epoch: 285\n",
      "**********\n",
      "Loss: 1.294060\n",
      "epoch: 286\n",
      "**********\n",
      "Loss: 1.289977\n",
      "epoch: 287\n",
      "**********\n",
      "Loss: 1.285891\n",
      "epoch: 288\n",
      "**********\n",
      "Loss: 1.281890\n",
      "epoch: 289\n",
      "**********\n",
      "Loss: 1.277893\n",
      "epoch: 290\n",
      "**********\n",
      "Loss: 1.273939\n",
      "epoch: 291\n",
      "**********\n",
      "Loss: 1.270013\n",
      "epoch: 292\n",
      "**********\n",
      "Loss: 1.266148\n",
      "epoch: 293\n",
      "**********\n",
      "Loss: 1.262317\n",
      "epoch: 294\n",
      "**********\n",
      "Loss: 1.258491\n",
      "epoch: 295\n",
      "**********\n",
      "Loss: 1.254711\n",
      "epoch: 296\n",
      "**********\n",
      "Loss: 1.251013\n",
      "epoch: 297\n",
      "**********\n",
      "Loss: 1.247293\n",
      "epoch: 298\n",
      "**********\n",
      "Loss: 1.243621\n",
      "epoch: 299\n",
      "**********\n",
      "Loss: 1.239992\n",
      "epoch: 300\n",
      "**********\n",
      "Loss: 1.236403\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(300):\n",
    "    print('epoch: {}'.format(epoch+1))\n",
    "    print('*'*10)\n",
    "    running_loss = 0\n",
    "    for data in trigram:\n",
    "        word, label = data\n",
    "        word = torch.LongTensor([word_to_idx[i] for i in word])\n",
    "        label = torch.LongTensor([word_to_idx[label]])\n",
    "        # forward\n",
    "        out = ngrammodel(word)\n",
    "        loss = criterion(out, label)\n",
    "        running_loss += loss.data[0]\n",
    "        # backward\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "    print('Loss: {:.6f}'.format(running_loss / len(word_to_idx)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 检测模型效果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "real word is thy, predict word is thy\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/envs/pytorch/lib/python3.6/site-packages/ipykernel/__main__.py:23: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n"
     ]
    }
   ],
   "source": [
    "word, label = trigram[3]\n",
    "word = torch.LongTensor([word_to_idx[i] for i in word])\n",
    "out = ngrammodel(word)\n",
    "_, predict_label = torch.max(out, 1)\n",
    "predict_word = idx_to_word[predict_label.item()]\n",
    "print('real word is {}, predict word is {}'.format(label, predict_word))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-  注：除了使用预训练词向量外，其余部分大部分直接使用了 https://ptorch.com/news/12.html 这篇博客中构建语言模型的代码。"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:pytorch]",
   "language": "python",
   "name": "conda-env-pytorch-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
